pull: cache summary and summary.sig

It allows an optimization to skip the download of the summary file
if its .sig file is unchanged.

Downloading the .sig file is much cheaper than downloading the summary
file from repositories with many branches.

https://bugzilla.gnome.org/show_bug.cgi?id=762973

Signed-off-by: Giuseppe Scrivano <gscrivan@redhat.com>

diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h
index 463b3dd75d7517b53c37581b3194a3cf76dc7645..484a6ec87e1263a6eab5461906f30b6cea676d41 100644 (file)
--- a/src/libostree/ostree-repo-private.h
+++ b/src/libostree/ostree-repo-private.h
@@ -33,6 +33,8 @@ G_BEGIN_DECLS
 
 #define _OSTREE_OBJECT_SIZES_ENTRY_SIGNATURE "ay"
 
+#define _OSTREE_SUMMARY_CACHE_PATH "tmp/cache/summaries"
+
 /**
  * OstreeRepo:
  *
@@ -316,5 +318,23 @@ _ostree_repo_read_bare_fd (OstreeRepo           *self,
 gboolean
 _ostree_repo_update_mtime (OstreeRepo        *self,
                            GError           **error);
-                           
+
+/* Load the summary from the cache if the provided .sig file is the same as the
+   cached version.  */
+gboolean
+_ostree_repo_load_cache_summary_if_same_sig (OstreeRepo        *self,
+                                             const char        *remote,
+                                             GBytes            *summary_sig,
+                                             GBytes            **summary,
+                                             GCancellable      *cancellable,
+                                             GError           **error);
+
+gboolean
+_ostree_repo_cache_summary (OstreeRepo        *self,
+                            const char        *remote,
+                            GBytes            *summary,
+                            GBytes            *summary_sig,
+                            GCancellable      *cancellable,
+                            GError           **error);
+
 G_END_DECLS

diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c
index 0be1b3802d780d7712d75da861661aee787bb0aa..9087848428548cce3feb8afe14a2e6684007a1f3 100644 (file)
--- a/src/libostree/ostree-repo-pull.c
+++ b/src/libostree/ostree-repo-pull.c
@@ -28,6 +28,7 @@
 #include "ostree-repo-static-delta-private.h"
 #include "ostree-metalink.h"
 #include "otutil.h"
+#include "ot-fs-utils.h"
 
 #include <gio/gunixinputstream.h>
 
@@ -1767,6 +1768,102 @@ ostree_repo_pull_one_dir (OstreeRepo               *self,
                                         progress, cancellable, error);
 }
 
+gboolean
+_ostree_repo_load_cache_summary_if_same_sig (OstreeRepo        *self,
+                                             const char        *remote,
+                                             GBytes            *summary_sig,
+                                             GBytes            **summary,
+                                             GCancellable      *cancellable,
+                                             GError           **error)
+{
+  gboolean ret = FALSE;
+  const char *summary_cache_sig_file = glnx_strjoina (_OSTREE_SUMMARY_CACHE_PATH, "/", remote, ".sig");
+
+  glnx_fd_close int prev_fd = -1;
+  g_autoptr(GBytes) old_sig_contents = NULL;
+
+  if (!ot_openat_ignore_enoent (self->repo_dir_fd, summary_cache_sig_file, &prev_fd, error))
+    goto out;
+
+  if (prev_fd < 0)
+    {
+      ret = TRUE;
+      goto out;
+    }
+
+  old_sig_contents = glnx_fd_readall_bytes (prev_fd, cancellable, error);
+  if (!old_sig_contents)
+    goto out;
+
+  if (g_bytes_compare (old_sig_contents, summary_sig) == 0)
+    {
+      const char *summary_cache_file = glnx_strjoina (_OSTREE_SUMMARY_CACHE_PATH, "/", remote);
+      glnx_fd_close int summary_fd = -1;
+      GBytes *summary_data;
+
+
+      summary_fd = openat (self->repo_dir_fd, summary_cache_file, O_CLOEXEC | O_RDONLY);
+      if (summary_fd < 0)
+        {
+          if (errno == ENOENT)
+            {
+              (void) unlinkat (self->repo_dir_fd, summary_cache_sig_file, 0);
+              ret = TRUE;
+              goto out;
+            }
+
+          glnx_set_error_from_errno (error);
+          goto out;
+        }
+
+      summary_data = glnx_fd_readall_bytes (summary_fd, cancellable, error);
+      if (!summary_data)
+        goto out;
+      *summary = summary_data;
+    }
+  ret = TRUE;
+
+ out:
+  return ret;
+}
+
+gboolean
+_ostree_repo_cache_summary (OstreeRepo        *self,
+                            const char        *remote,
+                            GBytes            *summary,
+                            GBytes            *summary_sig,
+                            GCancellable      *cancellable,
+                            GError           **error)
+{
+  gboolean ret = FALSE;
+  const char *summary_cache_file = glnx_strjoina (_OSTREE_SUMMARY_CACHE_PATH, "/", remote);
+  const char *summary_cache_sig_file = glnx_strjoina (_OSTREE_SUMMARY_CACHE_PATH, "/", remote, ".sig");
+
+  if (!glnx_shutil_mkdir_p_at (self->repo_dir_fd, _OSTREE_SUMMARY_CACHE_PATH, 0775, cancellable, error))
+    goto out;
+
+  if (!glnx_file_replace_contents_at (self->repo_dir_fd,
+                                      summary_cache_file,
+                                      g_bytes_get_data (summary, NULL),
+                                      g_bytes_get_size (summary),
+                                      self->disable_fsync ? GLNX_FILE_REPLACE_NODATASYNC : GLNX_FILE_REPLACE_DATASYNC_NEW,
+                                      cancellable, error))
+    goto out;
+
+  if (!glnx_file_replace_contents_at (self->repo_dir_fd,
+                                      summary_cache_sig_file,
+                                      g_bytes_get_data (summary_sig, NULL),
+                                      g_bytes_get_size (summary_sig),
+                                      self->disable_fsync ? GLNX_FILE_REPLACE_NODATASYNC : GLNX_FILE_REPLACE_DATASYNC_NEW,
+                                      cancellable, error))
+    goto out;
+
+  ret = TRUE;
+ out:
+  return ret;
+
+}
+
 /* Documented in ostree-repo.c */
 gboolean
 ostree_repo_pull_with_options (OstreeRepo             *self,
@@ -1995,15 +2092,36 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
     g_autoptr(GVariant) refs = NULL;
     g_autoptr(GVariant) deltas = NULL;
     g_autoptr(GVariant) additional_metadata = NULL;
-      
-    if (!pull_data->summary)
+    gboolean summary_from_cache = FALSE;
+
+    if (!pull_data->summary_data_sig)
+      {
+        uri = suburi_new (pull_data->base_uri, "summary.sig", NULL);
+        if (!fetch_uri_contents_membuf_sync (pull_data, uri, FALSE, TRUE,
+                                             &bytes_sig, cancellable, error))
+          goto out;
+        soup_uri_free (uri);
+      }
+
+    if (bytes_sig && !_ostree_repo_load_cache_summary_if_same_sig (self,
+                                                                   remote_name_or_baseurl,
+                                                                   bytes_sig,
+                                                                   &bytes_summary,
+                                                                   cancellable,
+                                                                   error))
+      goto out;
+
+    if (bytes_summary)
+      summary_from_cache = TRUE;
+
+    if (!pull_data->summary && !bytes_summary)
       {
         uri = suburi_new (pull_data->base_uri, "summary", NULL);
         if (!fetch_uri_contents_membuf_sync (pull_data, uri, FALSE, TRUE,
                                              &bytes_summary, cancellable, error))
           goto out;
         soup_uri_free (uri);
-     }
+      }
 
     if (!bytes_summary && pull_data->gpg_verify_summary)
       {
@@ -2019,15 +2137,6 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
         goto out;
       }
 
-    if (bytes_summary)
-      {
-        uri = suburi_new (pull_data->base_uri, "summary.sig", NULL);
-        if (!fetch_uri_contents_membuf_sync (pull_data, uri, FALSE, TRUE,
-                                             &bytes_sig, cancellable, error))
-          goto out;
-        soup_uri_free (uri);
-      }
-
     if (!bytes_sig && pull_data->gpg_verify_summary)
       {
         g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
@@ -2044,6 +2153,18 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
           pull_data->summary_data_sig = g_bytes_ref (bytes_sig);
       }
 
+
+    if (!summary_from_cache && bytes_summary && bytes_sig)
+      {
+        if (!_ostree_repo_cache_summary (self,
+                                         remote_name_or_baseurl,
+                                         bytes_summary,
+                                         bytes_sig,
+                                         cancellable,
+                                         error))
+          goto out;
+      }
+
     if (pull_data->gpg_verify_summary && bytes_summary && bytes_sig)
       {
         g_autoptr(GVariant) sig_variant = NULL;

diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c
index 2398c46f1e0574fb000577ec21fdc4eb713f3009..25cfcb80d84cce6c823fc160175fa1505e3c2b51 100644 (file)
--- a/src/libostree/ostree-repo.c
+++ b/src/libostree/ostree-repo.c
@@ -1795,6 +1795,17 @@ repo_remote_fetch_summary (OstreeRepo    *self,
         goto out;
     }
 
+  if (*out_summary && *out_signatures)
+    {
+      if (!_ostree_repo_cache_summary (self,
+                                       name,
+                                       *out_summary,
+                                       *out_signatures,
+                                       cancellable,
+                                       error))
+        goto out;
+    }
+
   ret = TRUE;
 
  out: